#!/usr/bin/python
# Copyright 2007 Alex K (wtwf.com) All rights reserved.
# $Id: feedMaker,v 1.6 2008-05-07 21:24:50 ark Exp $

"""
Makes rss feeds from urls
"""

import logging
import os
import re
import sys
import traceback
import urllib
import urlparse

from xml.dom import minidom

from expandrss import feedExpander

if sys.version_info < (2, 4):
  raise "You probably need python 2.5 or greater"

outputDir = os.path.expanduser("~/html/wtwf/rss")

_HTML_ELEMENT_RE = re.compile("<[^>]*>")

class FeedMaker:
  def __init__(self, filename, url, title,
               description=None,
               fetchcommand=None,
               ):
    self.filename = filename
    self.url = url
    self.title = title
    if description:
      self.description = description
    else:
      self.description = title
    self.fetchcommand = fetchcommand



class AllLinksFeedMaker(FeedMaker):
  def __init__(self, filename, url, title,
               description=None,
               fetchcommand=None,
               endstring=None,
               max_items=10):
    FeedMaker.__init__(self, filename, url, title, description, fetchcommand)
    self.endstring = endstring
    self.max_items = max_items
    self.fe = feedExpander("", "", "<body", "</body", encoding='ascii',
                           replace=[("<blockquote>", ""),
                                    ('<blockquote class="style1">', ""),
                                    ("</blockquote>", ""),
                                    ])

  def create(self):
    """make the feed"""

    print "Opening: %s" % self.url

    if self.fetchcommand:
      urlh = os.popen(self.fetchcommand % self.url)
    else:
      urlh = urllib.urlopen(self.url)

    body = []
    for line in urlh.readlines():
      # for some reason the character 13 caused us trouble
      line = line.replace("%c" % 13, " ")
      body.append(line.strip(" \t\r\n%c" % 13))
    text = " ".join(body)
    urlh.close()

    # split on <a
    urls = re.split('<[aA]\s', text)
    logging.debug("There are %d urls", len(urls))
    hrefre = re.compile("""href=([^"']\S*|"[^"]*"|'[^']*')[^>]*>([^<]*)<""",
                        re.MULTILINE | re.IGNORECASE)
    # " an array of (href, linktext)
    items = []
    for url in urls:
      # find the href stuff
      result = hrefre.match(url)
      if result:
        desturl = result.group(1)
        if not os.path.splitext(desturl)[1].lower() in (".html", ".htm"):
          continue
        logging.debug("match %s - %s", desturl, result.group(2))
        items.append((urlparse.urljoin(self.url, result.group(1).strip("\"'")),
                                       result.group(2)))
        if len(items) == self.max_items:
          break
      if self.endstring and url.find(self.endstring) != -1:
        break

    # now create the rss xml
    document = minidom.getDOMImplementation().createDocument(None,
                                                           "rss",
                                                           None)
    top = document.documentElement
    top.setAttribute("version", "2.0")
    #text = document.createTextNode('Some textual content.')
    # top_element.appendChild(text)
    channel = document.createElement("channel")
    if self.title:
      title = document.createElement("title")
      title.appendChild(document.createTextNode(stripHtml(self.title)))
      channel.appendChild(title)
    link = document.createElement("link")
    link.appendChild(document.createTextNode(self.url))
    if self.description:
      desc = document.createElement("description")
      desc.appendChild(document.createTextNode(self.description))
      channel.appendChild(desc)
    channel.appendChild(link)

    # now each item
    guids = []
    for (href, linktext) in items:
      item = document.createElement("item")
      title = document.createElement("title")
      title.appendChild(document.createTextNode(stripHtml(linktext)))
      item.appendChild(title)
      link = document.createElement("link")
      link.appendChild(document.createTextNode(href))
      item.appendChild(link)
      desc = document.createElement("description")
      desc.appendChild(document.createTextNode(linktext))
      item.appendChild(desc)
      # if I was good I'd find the pub date from last modified

      # I'd also find the guid too
      guid = document.createElement("guid")
      while href in guids:
        href = href + "_"
      guids.append(href)
      guid.appendChild(document.createTextNode(href))
      item.appendChild(guid)

      # now expand this item to make it full
      self.fe.enhance(item, document)
      channel.appendChild(item)

    top.appendChild(channel)


    destination = os.path.join(outputDir, self.filename)
    logging.info("Writing doc: %s" % destination)
    fh = open(destination, "w")
    fh.write(document.toxml(encoding='utf-8'))
    fh.close()


class PageContentFeedMaker(FeedMaker):
  def __init__(self, filename, url, title,
               description=None,
               fetchcommand=None,
               beginstring=None,
               includebegin=False,
               endstring=None,
               includeend=False,
               sepstring=None,
               max_items=10):
    FeedMaker.__init__(self, filename, url, title, description, fetchcommand)
    self.beginstring = beginstring
    self.includebegin = includebegin
    self.sepstring = sepstring
    self.endstring = endstring
    self.includeend = includeend
    self.max_items = max_items

  def create(self):
    """make the feed"""

    print "Opening: %s" % self.url

    if self.fetchcommand:
      urlh = os.popen(self.fetchcommand % self.url)
    else:
      urlh = urllib.urlopen(self.url)

    body = []
    for line in urlh.readlines():
      body.append(line.strip(" \t\r\n%c" % 13))
    urlh.close()
    # now make each line end with <br>
    text = "<BR>\n".join((" ".join(body)).split("<BR>"))

    # strip out beginning and end
    if self.beginstring:
      idx = text.find(self.beginstring)
      if idx > 0:
        if self.includebegin:
          text = text[idx:]
        else:
          text = text[idx + len(self.beginstring):]
      else:
        logging.error("Failed to find begin string '%s' in:\n%s",
                      self.beginstring, text)
        return
    if self.endstring:
      idx = text.rfind(self.endstring)
      if idx > 0:
        if self.includeend:
          text = text[0:idx + len(self.endstring)]
        else:
          text = text[0:idx]
      else:
        logging.error("Failed to find end string '%s' in:\n%s",
                      self.endstring, text)
        return


    # now split on the sepstring
    stories = text.split(self.sepstring)
    logging.debug("There are %d stories", len(stories))

    # now create the rss xml
    document = minidom.getDOMImplementation().createDocument(None,
                                                           "rss",
                                                           None)
    top = document.documentElement
    top.setAttribute("version", "2.0")
    #text = document.createTextNode('Some textual content.')
    # top_element.appendChild(text)
    channel = document.createElement("channel")
    if self.title:
      title = document.createElement("title")
      title.appendChild(document.createTextNode(stripHtml(self.title)))
      channel.appendChild(title)
    link = document.createElement("link")
    link.appendChild(document.createTextNode(self.url))
    if self.description:
      desc = document.createElement("description")
      desc.appendChild(document.createTextNode(self.description))
      channel.appendChild(desc)
    channel.appendChild(link)

    # now each item
    guids = []

    for story in stories:
      if len(guids) == self.max_items:
        break
      story = story.strip()

      # the guidstr is the first line
      guidstr = ""
      titlestr = ""
      idx = 0
      while idx < len(story) and (story[idx] != "\n" and story[idx] != "/"):
        idx += 1

      titlestr = removeElements(story[0:idx])
      guidstr = "".join([onlyalpha(titlestr[x:x + 1]) for x in range(0, len(titlestr))])

      logging.info("'%s': '%s'", titlestr, guidstr)

      item = document.createElement("item")
      title = document.createElement("title")
      title.appendChild(document.createTextNode(titlestr))
      item.appendChild(title)
      link = document.createElement("link")
      link.appendChild(document.createTextNode(self.url))
      item.appendChild(link)
      desc = document.createElement("description")
      desc.appendChild(document.createTextNode(story))
      item.appendChild(desc)
      # if I was good I'd find the pub date from last modified

      # I'd also find the guid too
      guid = document.createElement("guid")
      while guidstr in guids:
        guidstr = guidstr + "_"
      guids.append(guidstr)
      guid.appendChild(document.createTextNode("%s?guid=%s" %
                                               (self.url, guidstr)))
      item.appendChild(guid)

      channel.appendChild(item)

    top.appendChild(channel)

    destination = os.path.join(outputDir, self.filename)
    logging.info("Writing doc: %s" % destination)
    fh = open(destination, "w")
    fh.write(document.toxml(encoding='utf-8'))
    fh.close()

def onlyalpha(txt):
  if txt.isalnum():
    return txt
  else:
    return ""

def removeElements(txt):
  """Remove all HTML type elements (dumb)"""
  return re.sub(_HTML_ELEMENT_RE, "", txt)

def stripHtml(txt):
  txt = txt.replace("&amp;", "&")
  txt = txt.replace("&nbsp;", " ")
  txt = txt.replace("&lt;", "<")
  txt = txt.replace("&gt;", ">")
  return txt

def Main(args):

  userfile = os.path.expanduser("~/.feedmakerrc.py")
  if os.path.exists(userfile):
    logging.debug("Loading: %s", userfile)
    execfile(userfile)

  for feed in feeds:
    try:
      feed.create()
    except:
      logging.info("\n\nError with: %s\n", feed.url)
      traceback.print_exc()

  # now do some manual stuff
  for cmd in systemCmds:
    print cmd
    os.system(cmd)

if __name__ == '__main__':
  logging.basicConfig()
  logging.getLogger().setLevel(logging.DEBUG)
  logging.debug("Starting %s", " ".join(sys.argv))

  Main(sys.argv)
